<!DOCTYPE html><html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" data-whc_version="25.0">
    <head><link rel="shortcut icon" href="../../../oxygen-webhelp/template/images/favicon.png"/><link rel="icon" href="../../../oxygen-webhelp/template/images/favicon.png"/><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="copyright" content="(C) Copyright 2024"/><meta name="generator" content="DITA-OT"/><meta name="description" content="Supported pipeline types: Data Collector The Spark Evaluator performs custom processing within a pipeline based on a Spark application that you develop. For information about supported versions, see ..."/><meta name="prodname" content="Data Collector"/><meta name="version" content="3"/><meta name="release" content="16"/><meta name="modification" content="0"/>        
      <title>Spark Evaluator (deprecated)</title><!--  Generated with Oxygen version 25.1, build number 2023042410.  --><meta name="wh-path2root" content="../../../"/><meta name="wh-toc-id" content="concept_cpx_1lm_zx-d16893e41717"/><meta name="wh-source-relpath" content="datacollector/UserGuide/Processors/Spark.dita"/><meta name="wh-out-relpath" content="datacollector/UserGuide/Processors/Spark.html"/>

    <link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/app/commons.css?buildId=2023042410"/>
    <link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/app/topic.css?buildId=2023042410"/>

    <script src="../../../oxygen-webhelp/app/options/properties.js?buildId=20240802104629"></script>
    <script src="../../../oxygen-webhelp/app/localization/strings.js?buildId=2023042410"></script>
    <script src="../../../oxygen-webhelp/app/search/index/keywords.js?buildId=20240802104629"></script>
    <script defer="defer" src="../../../oxygen-webhelp/app/commons.js?buildId=2023042410"></script>
    <script defer="defer" src="../../../oxygen-webhelp/app/topic.js?buildId=2023042410"></script>
<link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/template/light.css?buildId=2023042410"/><link rel="stylesheet" type="text/css" href="../../../skin.css"/></head>

    <body class="wh_topic_page frmBody">
        
        
        

        
<nav class="navbar navbar-default wh_header" data-whc_version="25.0">
    <div class="container-fluid">
        <div class="wh_header_flex_container navbar-nav navbar-expand-md navbar-dark">
            <div class="wh_logo_and_publication_title_container">
                <div class="wh_logo_and_publication_title">
                    
                    <!--
                            This component will be generated when the next parameters are specified in the transformation scenario:
                            'webhelp.logo.image' and 'webhelp.logo.image.target.url'.
                            See: http://oxygenxml.com/doc/versions/17.1/ug-editor/#topics/dita_webhelp_output.html.
                    -->
                    
                    <div class=" wh_publication_title "><a href="../../../index.html"><span class="booktitle">  <span class="ph mainbooktitle"><span class="ph">Data Collector</span> User Guide</span>  </span></a></div>
                    
                </div>
                
                <!-- The menu button for mobile devices is copied in the output only when the 'webhelp.show.top.menu' parameter is set to 'yes' -->
                
            </div>

            <div class="wh_top_menu_and_indexterms_link collapse navbar-collapse">
                
                
                <div class=" wh_indexterms_link "><a href="../../../indexTerms.html" title="Index" aria-label="Go to index terms page"><span>Index</span></a></div>
                
            </div>
        </div>
    </div>
</nav>

        <div class=" wh_search_input navbar-form wh_topic_page_search search " role="form">


<form id="searchForm" method="get" role="search" action="../../../search.html"><div><input type="search" placeholder="Search " class="wh_search_textfield" id="textToSearch" name="searchQuery" aria-label="Search query" required="required"/><button type="submit" class="wh_search_button" aria-label="Search"><span class="search_input_text">Search</span></button></div></form>

</div>
        
        <div class="container-fluid">
            <div class="row">

                <nav class="wh_tools d-print-none">
                    
<div data-tooltip-position="bottom" class=" wh_breadcrumb "><ol class="d-print-none"><li><span class="home"><a href="../../../index.html"><span>Home</span></a></span></li><li><div class="topicref" data-id="concept_yjl_nc5_jq"><div class="title"><a href="../../../datacollector/UserGuide/Processors/Processors_title.html">Processors</a></div></div></li><li class="active"><div class="topicref" data-id="concept_cpx_1lm_zx"><div class="title"><a href="../../../datacollector/UserGuide/Processors/Spark.html#concept_cpx_1lm_zx">Spark Evaluator (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li></ol></div>



                    <div class="wh_right_tools "><button class="wh_hide_highlight" aria-label="Toggle search highlights" title="Toggle search highlights"></button><button class="webhelp_expand_collapse_sections" data-next-state="collapsed" aria-label="Collapse sections" title="Collapse sections"></button><div class=" wh_navigation_links "><span id="topic_navigation_links" class="navheader">
  
<span class="navprev"><a class="- topic/link link" href="../../../datacollector/UserGuide/Processors/SchemaGenerator.html#concept_rfz_ks3_x1b" title="Schema Generator" aria-label="Previous topic: Schema Generator" rel="prev"></a></span>  
<span class="navnext"><a class="- topic/link link" href="../../../datacollector/UserGuide/Processors/SQLParser.html#concept_zh2_kfj_tdb" title="SQL Parser" aria-label="Next topic: SQL Parser" rel="next"></a></span>  </span></div>
<!--External resource link-->
<div class=" wh_print_link print d-none d-md-inline-block "><button onClick="window.print()" title="Print this page" aria-label="Print this page"></button></div>
                        
                        
                        
                        
                    </div>
                </nav>
            </div>

            

<div class="wh_content_area">
                <div class="row">
                    


                        <nav role="navigation" id="wh_publication_toc" class="col-lg-3 col-md-3 col-sm-12 d-md-block d-none d-print-none">
<div id="wh_publication_toc_content">


                            <div class=" wh_publication_toc " data-tooltip-position="right"><span class="expand-button-action-labels"><span id="button-expand-action" role="button" aria-label="Expand"></span><span id="button-collapse-action" role="button" aria-label="Collapse"></span><span id="button-pending-action" role="button" aria-label="Pending"></span></span><ul role="tree" aria-label="Table of Contents"><li role="treeitem" aria-expanded="false"><div data-tocid="concept_htw_ghg_jq-d16893e53" class="topicref" data-id="concept_htw_ghg_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_htw_ghg_jq-d16893e53-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Getting_Started/GettingStarted_Title.html#concept_htw_ghg_jq" id="concept_htw_ghg_jq-d16893e53-link">Getting Started</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_l2v_nlp_mpb-d16893e331" class="topicref" data-id="concept_l2v_nlp_mpb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_l2v_nlp_mpb-d16893e331-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/ReleaseNotes/ReleaseNotes.html#concept_l2v_nlp_mpb" id="concept_l2v_nlp_mpb-d16893e331-link">Release Notes</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_l4q_flb_kr-d16893e2582" class="topicref" data-id="concept_l4q_flb_kr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_l4q_flb_kr-d16893e2582-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Installation/Install_title.html" id="concept_l4q_flb_kr-d16893e2582-link">Installation</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ylh_yyz_ky-d16893e3984" class="topicref" data-id="concept_ylh_yyz_ky" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ylh_yyz_ky-d16893e3984-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Configuration/Config_title.html" id="concept_ylh_yyz_ky-d16893e3984-link">Configuration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ejk_f1f_5v-d16893e7058" class="topicref" data-id="concept_ejk_f1f_5v" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ejk_f1f_5v-d16893e7058-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Upgrade/Upgrade_title.html" id="concept_ejk_f1f_5v-d16893e7058-link">Upgrade</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qsw_cjy_bt-d16893e10103" class="topicref" data-id="concept_qsw_cjy_bt" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qsw_cjy_bt-d16893e10103-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Design/PipelineDesign_title.html" id="concept_qsw_cjy_bt-d16893e10103-link">Pipeline Concepts and Design</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qn1_wn4_kq-d16893e11199" class="topicref" data-id="concept_qn1_wn4_kq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qn1_wn4_kq-d16893e11199-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Configuration/PipelineConfiguration_title.html" id="concept_qn1_wn4_kq-d16893e11199-link">Pipeline Configuration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hdr_gyw_41b-d16893e13057" class="topicref" data-id="concept_hdr_gyw_41b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hdr_gyw_41b-d16893e13057-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Data_Formats/DataFormats-Title.html" id="concept_hdr_gyw_41b-d16893e13057-link">Data Formats</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yjl_nc5_jq-d16893e14164" class="topicref" data-id="concept_yjl_nc5_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yjl_nc5_jq-d16893e14164-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Origins/Origins_title.html" id="concept_yjl_nc5_jq-d16893e14164-link">Origins</a></div></div></li><li role="treeitem" aria-expanded="true"><div data-tocid="concept_yjl_nc5_jq-d16893e35197" class="topicref" data-id="concept_yjl_nc5_jq" data-state="expanded"><span role="button" tabindex="0" aria-labelledby="button-collapse-action concept_yjl_nc5_jq-d16893e35197-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Processors_title.html" id="concept_yjl_nc5_jq-d16893e35197-link">Processors</a></div></div><ul role="group" class="navbar-nav nav-list"><li role="treeitem"><div data-tocid="concept_hpr_twm_jq-d16893e35219" class="topicref" data-id="concept_hpr_twm_jq" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Processors_overview.html#concept_hpr_twm_jq" id="concept_hpr_twm_jq-d16893e35219-link">Processors</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ujj_spy_kv-d16893e35243" class="topicref" data-id="concept_ujj_spy_kv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ujj_spy_kv-d16893e35243-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Base64Decoder.html#concept_ujj_spy_kv" id="concept_ujj_spy_kv-d16893e35243-link">Base64 Field Decoder</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wtr_mpy_kv-d16893e35289" class="topicref" data-id="concept_wtr_mpy_kv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wtr_mpy_kv-d16893e35289-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Base64Encoder.html#concept_wtr_mpy_kv" id="concept_wtr_mpy_kv-d16893e35289-link">Base64 Field Encoder</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_akz_zsr_2jb-d16893e35335" class="topicref" data-id="concept_akz_zsr_2jb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_akz_zsr_2jb-d16893e35335-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/ControlHubAPI.html#concept_akz_zsr_2jb" id="concept_akz_zsr_2jb-d16893e35335-link">Control Hub API</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_rxk_1dq_2fb-d16893e35502" class="topicref" data-id="concept_rxk_1dq_2fb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_rxk_1dq_2fb-d16893e35502-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/CouchbaseLookup.html#concept_rxk_1dq_2fb" id="concept_rxk_1dq_2fb-d16893e35502-link">Couchbase Lookup</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hw1_gq4_3fb-d16893e35572" class="topicref" data-id="concept_hw1_gq4_3fb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hw1_gq4_3fb-d16893e35572-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/DataGenerator.html#concept_hw1_gq4_3fb" id="concept_hw1_gq4_3fb-d16893e35572-link">Data Generator</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xw3_4xk_r1b-d16893e35663" class="topicref" data-id="concept_xw3_4xk_r1b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xw3_4xk_r1b-d16893e35663-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/DataParser.html#concept_xw3_4xk_r1b" id="concept_xw3_4xk_r1b-d16893e35663-link">Data Parser</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ez5_pvf_wbb-d16893e35731" class="topicref" data-id="concept_ez5_pvf_wbb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ez5_pvf_wbb-d16893e35731-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Delay.html#concept_ez5_pvf_wbb" id="concept_ez5_pvf_wbb-d16893e35731-link">Delay</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_zs3_vfk_hfb-d16893e35777" class="topicref" data-id="concept_zs3_vfk_hfb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_zs3_vfk_hfb-d16893e35777-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/EncryptDecrypt.html#concept_zs3_vfk_hfb" id="concept_zs3_vfk_hfb-d16893e35777-link">Encrypt and Decrypt Fields</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_zm2_pp3_wq-d16893e35995" class="topicref" data-id="concept_zm2_pp3_wq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_zm2_pp3_wq-d16893e35995-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Expression.html#concept_zm2_pp3_wq" id="concept_zm2_pp3_wq-d16893e35995-link">Expression Evaluator</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_njn_3kk_fx-d16893e36108" class="topicref" data-id="concept_njn_3kk_fx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_njn_3kk_fx-d16893e36108-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldFlattener.html#concept_njn_3kk_fx" id="concept_njn_3kk_fx-d16893e36108-link">Field Flattener</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ivv_c3k_wq-d16893e36198" class="topicref" data-id="concept_ivv_c3k_wq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ivv_c3k_wq-d16893e36198-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldHasher.html#concept_ivv_c3k_wq" id="concept_ivv_c3k_wq-d16893e36198-link">Field Hasher</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_q5y_tdq_xgb-d16893e36316" class="topicref" data-id="concept_q5y_tdq_xgb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_q5y_tdq_xgb-d16893e36316-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldMapper.html#concept_q5y_tdq_xgb" id="concept_q5y_tdq_xgb-d16893e36316-link">Field Mapper</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hjc_t4k_wq-d16893e36460" class="topicref" data-id="concept_hjc_t4k_wq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hjc_t4k_wq-d16893e36460-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldMasker.html#concept_hjc_t4k_wq" id="concept_hjc_t4k_wq-d16893e36460-link">Field Masker</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pgm_tsl_gt-d16893e36531" class="topicref" data-id="concept_pgm_tsl_gt" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pgm_tsl_gt-d16893e36531-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldMerger.html#concept_pgm_tsl_gt" id="concept_pgm_tsl_gt-d16893e36531-link">Field Merger</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_krp_5fv_vy-d16893e36577" class="topicref" data-id="concept_krp_5fv_vy" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_krp_5fv_vy-d16893e36577-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldOrder.html#concept_krp_5fv_vy" id="concept_krp_5fv_vy-d16893e36577-link">Field Order</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ekg_313_qw-d16893e36648" class="topicref" data-id="concept_ekg_313_qw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ekg_313_qw-d16893e36648-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/ListPivoter.html#concept_ekg_313_qw" id="concept_ekg_313_qw-d16893e36648-link">Field Pivoter</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jdd_blr_wq-d16893e36719" class="topicref" data-id="concept_jdd_blr_wq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jdd_blr_wq-d16893e36719-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldRemover.html#concept_jdd_blr_wq" id="concept_jdd_blr_wq-d16893e36719-link">Field Remover</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_vyv_zsg_ht-d16893e36765" class="topicref" data-id="concept_vyv_zsg_ht" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_vyv_zsg_ht-d16893e36765-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldRenamer.html#concept_vyv_zsg_ht" id="concept_vyv_zsg_ht-d16893e36765-link">Field Renamer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_rw4_2d3_4cb-d16893e36842" class="topicref" data-id="concept_rw4_2d3_4cb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_rw4_2d3_4cb-d16893e36842-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldReplacer.html#concept_rw4_2d3_4cb" id="concept_rw4_2d3_4cb-d16893e36842-link">Field Replacer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_vlj_vph_yq-d16893e36960" class="topicref" data-id="concept_vlj_vph_yq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_vlj_vph_yq-d16893e36960-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldSplitter.html#concept_vlj_vph_yq" id="concept_vlj_vph_yq-d16893e36960-link">Field Splitter</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_is3_zkp_wq-d16893e37073" class="topicref" data-id="concept_is3_zkp_wq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_is3_zkp_wq-d16893e37073-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldTypeConverter.html#concept_is3_zkp_wq" id="concept_is3_zkp_wq-d16893e37073-link">Field Type Converter</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_o3b_t1k_yx-d16893e37194" class="topicref" data-id="concept_o3b_t1k_yx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_o3b_t1k_yx-d16893e37194-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/FieldZip.html#concept_o3b_t1k_yx" id="concept_o3b_t1k_yx-d16893e37194-link">Field Zip</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fch_fc3_ms-d16893e37306" class="topicref" data-id="concept_fch_fc3_ms" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fch_fc3_ms-d16893e37306-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/GeoIP.html#concept_fch_fc3_ms" id="concept_fch_fc3_ms-d16893e37306-link">Geo IP</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ldh_sct_gv-d16893e37440" class="topicref" data-id="concept_ldh_sct_gv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ldh_sct_gv-d16893e37440-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Groovy.html#concept_ldh_sct_gv" id="concept_ldh_sct_gv-d16893e37440-link">Groovy Evaluator</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_mnj_zhq_bw-d16893e37741" class="topicref" data-id="concept_mnj_zhq_bw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_mnj_zhq_bw-d16893e37741-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/HBaseLookup.html#concept_mnj_zhq_bw" id="concept_mnj_zhq_bw-d16893e37741-link">HBase Lookup</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_rz5_nft_zv-d16893e37912" class="topicref" data-id="concept_rz5_nft_zv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_rz5_nft_zv-d16893e37912-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/HiveMetadata.html#concept_rz5_nft_zv" id="concept_rz5_nft_zv-d16893e37912-link">Hive Metadata</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ghx_ypr_fw-d16893e38231" class="topicref" data-id="concept_ghx_ypr_fw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ghx_ypr_fw-d16893e38231-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/HTTPClient.html#concept_ghx_ypr_fw" id="concept_ghx_ypr_fw-d16893e38231-link">HTTP Client</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ghx_ypr_fw-d16893e38850" class="topicref" data-id="concept_ghx_ypr_fw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ghx_ypr_fw-d16893e38850-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/HTTPRouter.html#concept_ghx_ypr_fw" id="concept_ghx_ypr_fw-d16893e38850-link">HTTP Router</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_n2p_jgf_lr-d16893e38896" class="topicref" data-id="concept_n2p_jgf_lr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_n2p_jgf_lr-d16893e38896-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/JavaScript.html#concept_n2p_jgf_lr" id="concept_n2p_jgf_lr-d16893e38896-link">JavaScript Evaluator</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ysc_ccy_hw-d16893e39172" class="topicref" data-id="concept_ysc_ccy_hw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ysc_ccy_hw-d16893e39172-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/JDBCLookup.html#concept_ysc_ccy_hw" id="concept_ysc_ccy_hw-d16893e39172-link">JDBC Lookup</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qbx_lcy_hw-d16893e39495" class="topicref" data-id="concept_qbx_lcy_hw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qbx_lcy_hw-d16893e39495-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/JDBCTee.html#concept_qbx_lcy_hw" id="concept_qbx_lcy_hw-d16893e39495-link">JDBC Tee</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jmg_hw1_h1b-d16893e39677" class="topicref" data-id="concept_jmg_hw1_h1b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jmg_hw1_h1b-d16893e39677-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/JSONGenerator.html#concept_jmg_hw1_h1b" id="concept_jmg_hw1_h1b-d16893e39677-link">JSON Generator</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_bs1_4t3_yq-d16893e39725" class="topicref" data-id="concept_bs1_4t3_yq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_bs1_4t3_yq-d16893e39725-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/JSONParser.html#concept_bs1_4t3_yq" id="concept_bs1_4t3_yq-d16893e39725-link">JSON Parser</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_a1h_lkf_lr-d16893e39774" class="topicref" data-id="concept_a1h_lkf_lr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_a1h_lkf_lr-d16893e39774-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Jython.html#concept_a1h_lkf_lr" id="concept_a1h_lkf_lr-d16893e39774-link">Jython Evaluator</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_g4y_hkh_5xb-d16893e40074" class="topicref" data-id="concept_g4y_hkh_5xb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_g4y_hkh_5xb-d16893e40074-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/KaitaiStruct.html#concept_g4y_hkh_5xb" id="concept_g4y_hkh_5xb-d16893e40074-link">Kaitai Struct Parser</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_a1x_3wl_p1b-d16893e40162" class="topicref" data-id="concept_a1x_3wl_p1b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_a1x_3wl_p1b-d16893e40162-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/KuduLookup.html#concept_a1x_3wl_p1b" id="concept_a1x_3wl_p1b-d16893e40162-link">Kudu Lookup</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ulm_qdq_fs-d16893e40342" class="topicref" data-id="concept_ulm_qdq_fs" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ulm_qdq_fs-d16893e40342-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/LogParser.html#concept_ulm_qdq_fs" id="concept_ulm_qdq_fs-d16893e40342-link">Log Parser</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wnr_wlv_gfb-d16893e40413" class="topicref" data-id="concept_wnr_wlv_gfb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wnr_wlv_gfb-d16893e40413-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/MLeap.html#concept_wnr_wlv_gfb" id="concept_wnr_wlv_gfb-d16893e40413-link">MLeap Evaluator</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_t4t_myy_dyb-d16893e40529" class="topicref" data-id="concept_t4t_myy_dyb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_t4t_myy_dyb-d16893e40529-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/MongoDBAtlasLookup.html#concept_t4t_myy_dyb" id="concept_t4t_myy_dyb-d16893e40529-link">MongoDB Atlas Lookup</a><div class="wh-tooltip"><p class="shortdesc">The MongoDB Atlas Lookup processor performs lookups in MongoDB Atlas or MongoDB         Enterprise Server and passes all values from the returned document to a new list-map field         in the record. </p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_rrp_t4w_2fb-d16893e40664" class="topicref" data-id="concept_rrp_t4w_2fb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_rrp_t4w_2fb-d16893e40664-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/MongoDBLookup.html#concept_rrp_t4w_2fb" id="concept_rrp_t4w_2fb-d16893e40664-link">MongoDB Lookup</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_r3s_3fv_gfb-d16893e40850" class="topicref" data-id="concept_r3s_3fv_gfb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_r3s_3fv_gfb-d16893e40850-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/PMML.html#concept_r3s_3fv_gfb" id="concept_r3s_3fv_gfb-d16893e40850-link">PMML Evaluator</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_lcp_ssh_qcb-d16893e40992" class="topicref" data-id="concept_lcp_ssh_qcb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_lcp_ssh_qcb-d16893e40992-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/PostgreSQLMetadata.html#concept_lcp_ssh_qcb" id="concept_lcp_ssh_qcb-d16893e40992-link">PostgreSQL Metadata</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_z3m_v52_zq-d16893e41156" class="topicref" data-id="concept_z3m_v52_zq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_z3m_v52_zq-d16893e41156-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/RDeduplicator.html#concept_z3m_v52_zq" id="concept_z3m_v52_zq-d16893e41156-link">Record Deduplicator</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ng3_lpr_pv-d16893e41227" class="topicref" data-id="concept_ng3_lpr_pv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ng3_lpr_pv-d16893e41227-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/RedisLookup.html#concept_ng3_lpr_pv" id="concept_ng3_lpr_pv-d16893e41227-link">Redis Lookup</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_plq_xzc_zsb-d16893e41326" class="topicref" data-id="concept_plq_xzc_zsb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_plq_xzc_zsb-d16893e41326-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/SalesforceBulkLookup.html#concept_plq_xzc_zsb" id="concept_plq_xzc_zsb-d16893e41326-link">Salesforce Bulk API 2.0 Lookup</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_k23_3rk_yx-d16893e41444" class="topicref" data-id="concept_k23_3rk_yx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_k23_3rk_yx-d16893e41444-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/SalesforceLookup.html#concept_k23_3rk_yx" id="concept_k23_3rk_yx-d16893e41444-link">Salesforce Lookup</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_rfz_ks3_x1b-d16893e41605" class="topicref" data-id="concept_rfz_ks3_x1b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_rfz_ks3_x1b-d16893e41605-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/SchemaGenerator.html#concept_rfz_ks3_x1b" id="concept_rfz_ks3_x1b-d16893e41605-link">Schema Generator</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="true" class="active"><div data-tocid="concept_cpx_1lm_zx-d16893e41717" class="topicref" data-id="concept_cpx_1lm_zx" data-state="expanded"><span role="button" tabindex="0" aria-labelledby="button-collapse-action concept_cpx_1lm_zx-d16893e41717-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Spark.html#concept_cpx_1lm_zx" id="concept_cpx_1lm_zx-d16893e41717-link">Spark Evaluator (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div><ul role="group" class="navbar-nav nav-list"><li role="treeitem"><div data-tocid="concept_nbj_1jb_c1b-d16893e41741" class="topicref" data-id="concept_nbj_1jb_c1b" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Spark.html#concept_nbj_1jb_c1b" id="concept_nbj_1jb_c1b-d16893e41741-link">Spark Versions and Stage Libraries</a></div></div></li><li role="treeitem"><div data-tocid="concept_hpm_tns_hz-d16893e41763" class="topicref" data-id="concept_hpm_tns_hz" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Spark.html#concept_hpm_tns_hz" id="concept_hpm_tns_hz-d16893e41763-link">Standalone Pipelines</a></div></div></li><li role="treeitem"><div data-tocid="concept_b1l_5lv_jz-d16893e41785" class="topicref" data-id="concept_b1l_5lv_jz" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Spark.html#concept_b1l_5lv_jz" id="concept_b1l_5lv_jz-d16893e41785-link">Cluster Pipelines</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem"><div data-tocid="concept_lfl_dvd_1y-d16893e41809" class="topicref" data-id="concept_lfl_dvd_1y" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Spark.html#concept_lfl_dvd_1y" id="concept_lfl_dvd_1y-d16893e41809-link">Developing the Spark Application</a></div></div></li><li role="treeitem"><div data-tocid="task_dr2_gvd_1y-d16893e41831" class="topicref" data-id="task_dr2_gvd_1y" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Spark.html#task_dr2_gvd_1y" id="task_dr2_gvd_1y-d16893e41831-link">Installing the Application</a><div class="wh-tooltip"><p class="shortdesc">Install the Spark application JAR file as an external library for <span class="ph">Data Collector</span>. If         your custom Spark application depends on external libraries other than the <span class="ph">streamsets-datacollector-api,                         streamsets-datacollector-spark-api, and spark-core libraries</span>, install those libraries in the same location as well. </p></div></div></div></li><li role="treeitem"><div data-tocid="task_g1p_gqn_zx-d16893e41862" class="topicref" data-id="task_g1p_gqn_zx" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Spark.html#task_g1p_gqn_zx" id="task_g1p_gqn_zx-d16893e41862-link">Configuring a Spark Evaluator Processor</a></div></div></li></ul></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_zh2_kfj_tdb-d16893e41884" class="topicref" data-id="concept_zh2_kfj_tdb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_zh2_kfj_tdb-d16893e41884-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/SQLParser.html#concept_zh2_kfj_tdb" id="concept_zh2_kfj_tdb-d16893e41884-link">SQL Parser</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_irv_l5r_2jb-d16893e42131" class="topicref" data-id="concept_irv_l5r_2jb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_irv_l5r_2jb-d16893e42131-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/StartJob-P.html#concept_irv_l5r_2jb" id="concept_irv_l5r_2jb-d16893e42131-link">Start Jobs</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_bbc_cxr_2jb-d16893e42269" class="topicref" data-id="concept_bbc_cxr_2jb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_bbc_cxr_2jb-d16893e42269-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/StartPipe-P.html#concept_bbc_cxr_2jb" id="concept_bbc_cxr_2jb-d16893e42269-link">Start Pipelines (deprecated)</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_aqz_t4r_pv-d16893e42361" class="topicref" data-id="concept_aqz_t4r_pv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_aqz_t4r_pv-d16893e42361-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/StaticLookup.html#concept_aqz_t4r_pv" id="concept_aqz_t4r_pv-d16893e42361-link">Static Lookup</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_tqv_t5r_wq-d16893e42410" class="topicref" data-id="concept_tqv_t5r_wq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_tqv_t5r_wq-d16893e42410-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/StreamSelector.html#concept_tqv_t5r_wq" id="concept_tqv_t5r_wq-d16893e42410-link">Stream Selector</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_otg_csh_z2b-d16893e42507" class="topicref" data-id="concept_otg_csh_z2b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_otg_csh_z2b-d16893e42507-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/TensorFlow.html#concept_otg_csh_z2b" id="concept_otg_csh_z2b-d16893e42507-link">TensorFlow Evaluator</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_o5k_dmf_zq-d16893e42697" class="topicref" data-id="concept_o5k_dmf_zq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_o5k_dmf_zq-d16893e42697-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/ValueReplacer.html#concept_o5k_dmf_zq" id="concept_o5k_dmf_zq-d16893e42697-link">Value Replacer (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xv5_xbd_zlb-d16893e42831" class="topicref" data-id="concept_xv5_xbd_zlb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xv5_xbd_zlb-d16893e42831-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/WaitJob.html#concept_xv5_xbd_zlb" id="concept_xv5_xbd_zlb-d16893e42831-link">Wait for Jobs</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_kd3_qld_zlb-d16893e42921" class="topicref" data-id="concept_kd3_qld_zlb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_kd3_qld_zlb-d16893e42921-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/WaitPipe.html#concept_kd3_qld_zlb" id="concept_kd3_qld_zlb-d16893e42921-link">Wait for Pipelines (deprecated)</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_sgs_qdz_v1c-d16893e43013" class="topicref" data-id="concept_sgs_qdz_v1c" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_sgs_qdz_v1c-d16893e43013-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/WebClientP.html#concept_sgs_qdz_v1c" id="concept_sgs_qdz_v1c-d16893e43013-link">Web Client</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_nwg_rx4_l2b-d16893e43398" class="topicref" data-id="concept_nwg_rx4_l2b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_nwg_rx4_l2b-d16893e43398-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/WholeFileTransformer.html#concept_nwg_rx4_l2b" id="concept_nwg_rx4_l2b-d16893e43398-link">Whole File Transformer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ofb_svm_5bb-d16893e43672" class="topicref" data-id="concept_ofb_svm_5bb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ofb_svm_5bb-d16893e43672-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Aggregator.html#concept_ofb_svm_5bb" id="concept_ofb_svm_5bb-d16893e43672-link">Windowing Aggregator</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ck4_255_sv-d16893e43916" class="topicref" data-id="concept_ck4_255_sv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ck4_255_sv-d16893e43916-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/XMLFlattener.html#concept_ck4_255_sv" id="concept_ck4_255_sv-d16893e43916-link">XML Flattener</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_dtt_q5q_k5-d16893e43987" class="topicref" data-id="concept_dtt_q5q_k5" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_dtt_q5q_k5-d16893e43987-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/XMLParser.html#concept_dtt_q5q_k5" id="concept_dtt_q5q_k5-d16893e43987-link">XML Parser</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li></ul></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_agj_cfj_br-d16893e44037" class="topicref" data-id="concept_agj_cfj_br" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_agj_cfj_br-d16893e44037-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Destinations-title.html" id="concept_agj_cfj_br-d16893e44037-link">Destinations</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_umc_1lk_fx-d16893e56072" class="topicref" data-id="concept_umc_1lk_fx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_umc_1lk_fx-d16893e56072-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Executors/Executors-title.html" id="concept_umc_1lk_fx-d16893e56072-link">Executors</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xxd_f5r_kx-d16893e59696" class="topicref" data-id="concept_xxd_f5r_kx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xxd_f5r_kx-d16893e59696-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Event_Handling/EventFramework-Title.html#concept_xxd_f5r_kx" id="concept_xxd_f5r_kx-d16893e59696-link">Dataflow Triggers</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_zq5_pb4_flb-d16893e60134" class="topicref" data-id="concept_zq5_pb4_flb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_zq5_pb4_flb-d16893e60134-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/Solutions-title.html" id="concept_zq5_pb4_flb-d16893e60134-link">Solutions</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ugp_kwf_xw-d16893e61337" class="topicref" data-id="concept_ugp_kwf_xw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ugp_kwf_xw-d16893e61337-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/DPM/DPM_title.html" id="concept_ugp_kwf_xw-d16893e61337-link">StreamSets Control Hub</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fyf_gkq_4bb-d16893e62693" class="topicref" data-id="concept_fyf_gkq_4bb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fyf_gkq_4bb-d16893e62693-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Edge_Mode/EdgePipelines_title.html" id="concept_fyf_gkq_4bb-d16893e62693-link"><span class="ph">StreamSets Data Collector Edge</span></a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wwq_gxc_py-d16893e63980" class="topicref" data-id="concept_wwq_gxc_py" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wwq_gxc_py-d16893e63980-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Multithreaded_Pipelines/MultithreadedPipelines.html#concept_wwq_gxc_py" id="concept_wwq_gxc_py-d16893e63980-link">Multithreaded Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_gzw_tdm_p2b-d16893e64187" class="topicref" data-id="concept_gzw_tdm_p2b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_gzw_tdm_p2b-d16893e64187-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Microservice/Microservice_Title.html#concept_gzw_tdm_p2b" id="concept_gzw_tdm_p2b-d16893e64187-link">Microservice Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="Orchestrators_Title-d16893e64348" class="topicref" data-id="Orchestrators_Title" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action Orchestrators_Title-d16893e64348-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Orchestration_Pipelines/OrchestrationPipelines_Title.html#Orchestrators_Title" id="Orchestrators_Title-d16893e64348-link">Orchestration Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wr1_ktz_bt-d16893e64489" class="topicref" data-id="concept_wr1_ktz_bt" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wr1_ktz_bt-d16893e64489-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/RPC_Pipelines/SDC_RPCpipelines_title.html#concept_wr1_ktz_bt" id="concept_wr1_ktz_bt-d16893e64489-link">SDC RPC Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fpz_5r4_vs-d16893e64679" class="topicref" data-id="concept_fpz_5r4_vs" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fpz_5r4_vs-d16893e64679-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Cluster_Mode/ClusterPipelines_title.html" id="concept_fpz_5r4_vs-d16893e64679-link">Cluster Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jjk_23z_sq-d16893e65172" class="topicref" data-id="concept_jjk_23z_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jjk_23z_sq-d16893e65172-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Data_Preview/DataPreview_Title.html#concept_jjk_23z_sq" id="concept_jjk_23z_sq-d16893e65172-link">Data Preview</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pgk_brx_rr-d16893e65458" class="topicref" data-id="concept_pgk_brx_rr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pgk_brx_rr-d16893e65458-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Alerts/RulesAlerts_title.html#concept_pgk_brx_rr" id="concept_pgk_brx_rr-d16893e65458-link">Rules and Alerts</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_asx_fdz_sq-d16893e65960" class="topicref" data-id="concept_asx_fdz_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_asx_fdz_sq-d16893e65960-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Monitoring/PipelineMonitoring_title.html#concept_asx_fdz_sq" id="concept_asx_fdz_sq-d16893e65960-link">Pipeline Monitoring</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_o3l_dtr_5q-d16893e66304" class="topicref" data-id="concept_o3l_dtr_5q" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_o3l_dtr_5q-d16893e66304-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Maintenance/PipelineMaintenance_title.html#concept_o3l_dtr_5q" id="concept_o3l_dtr_5q-d16893e66304-link">Pipeline Maintenance</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yms_ftm_sq-d16893e66768" class="topicref" data-id="concept_yms_ftm_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yms_ftm_sq-d16893e66768-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Administration/Administration_title.html#concept_yms_ftm_sq" id="concept_yms_ftm_sq-d16893e66768-link">Administration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_nls_w1r_ks-d16893e67508" class="topicref" data-id="concept_nls_w1r_ks" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_nls_w1r_ks-d16893e67508-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/Tutorial-title.html" id="concept_nls_w1r_ks-d16893e67508-link">Tutorial</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_sh3_frm_tq-d16893e68001" class="topicref" data-id="concept_sh3_frm_tq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_sh3_frm_tq-d16893e68001-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Troubleshooting/Troubleshooting_title.html#concept_sh3_frm_tq" id="concept_sh3_frm_tq-d16893e68001-link">Troubleshooting</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xbx_rs1_tq-d16893e68798" class="topicref" data-id="concept_xbx_rs1_tq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xbx_rs1_tq-d16893e68798-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Glossary/Glossary_title.html#concept_xbx_rs1_tq" id="concept_xbx_rs1_tq-d16893e68798-link">Glossary</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jn1_nzb_kv-d16893e68843" class="topicref" data-id="concept_jn1_nzb_kv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jn1_nzb_kv-d16893e68843-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-DataFormats/DataFormat_Title.html#concept_jn1_nzb_kv" id="concept_jn1_nzb_kv-d16893e68843-link">Data Formats by Stage</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pvm_yt3_wq-d16893e68958" class="topicref" data-id="concept_pvm_yt3_wq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pvm_yt3_wq-d16893e68958-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Expression_Language/ExpressionLanguage_title.html" id="concept_pvm_yt3_wq-d16893e68958-link">Expression Language</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_vcj_1ws_js-d16893e69669" class="topicref" data-id="concept_vcj_1ws_js" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_vcj_1ws_js-d16893e69669-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-RegEx/RegEx-Title.html#concept_vcj_1ws_js" id="concept_vcj_1ws_js-d16893e69669-link">Regular Expressions</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_chv_vmj_wr-d16893e69787" class="topicref" data-id="concept_chv_vmj_wr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_chv_vmj_wr-d16893e69787-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-GrokPatterns/GrokPatterns_title.html#concept_chv_vmj_wr" id="concept_chv_vmj_wr-d16893e69787-link">Grok Patterns</a></div></div></li></ul></div>
                        

</div>
</nav>
                    


                    
                    <div id="wh_topic_body" class="col-lg-7 col-md-9 col-sm-12">
<button id="wh_close_publication_toc_button" class="close-toc-button d-none" aria-label="Toggle publishing table of content" aria-controls="wh_publication_toc" aria-expanded="true"><span class="close-toc-icon-container"><span class="close-toc-icon"></span></span></button><button id="wh_close_topic_toc_button" class="close-toc-button d-none" aria-label="Toggle topic table of content" aria-controls="wh_topic_toc" aria-expanded="true"><span class="close-toc-icon-container"><span class="close-toc-icon"></span></span></button>

                        
<div class=" wh_topic_content body "><main role="main"><article class="" role="article" aria-labelledby="ariaid-title1"><article class="nested0" aria-labelledby="ariaid-title1" id="concept_cpx_1lm_zx">
    <h1 class="- topic/title title topictitle1" id="ariaid-title1">Spark Evaluator (deprecated)</h1>
    
    <div class="- topic/body concept/conbody body conbody"><p class="- topic/shortdesc shortdesc"></p>
        <div class="- topic/p p"><div class="simpletable-container"><table class="- topic/simpletable simpletable frame-none" id="concept_cpx_1lm_zx__simpletable_k44_gfm_rgb" data-ofbid="concept_cpx_1lm_zx__simpletable_k44_gfm_rgb"><colgroup><col style="width:100%"/></colgroup><tbody><tr class="- topic/strow strow">
                <td class="- topic/stentry stentry"><a class="- topic/xref xref" href="../Pipeline_Configuration/ProductIcons_Doc.html#concept_mjg_ly5_pgb">Supported pipeline types:</a><ul class="- topic/ul ul" id="concept_cpx_1lm_zx__ul_s3x_b2m_rgb" data-ofbid="concept_cpx_1lm_zx__ul_s3x_b2m_rgb">
                        <li class="- topic/li li">
                            <p class="- topic/p p"><img class="- topic/image image" id="concept_cpx_1lm_zx__image_t3x_b2m_rgb" src="../../../reusable-content/datacollector/reusable-topics/../../shared-graphics/icon-SDC.png" height="21" width="21"/> Data Collector</p>
                        </li>
                    </ul></td>
            </tr></tbody></table></div>The Spark Evaluator performs custom processing
            within a pipeline based on a Spark application that you develop. <span class="- topic/ph ph">For information about supported versions, see <a class="- topic/xref xref" href="../Installation/SupportedSystemVersions.html#concept_s5h_bcr_n4b">Supported Systems and Versions</a>.</span><div class="- topic/note note important note_important"><span class="note__title">Important:</span> <span class="- topic/ph ph">This stage is <a class="- topic/xref xref" href="../Pipeline_Design/Deprecated.html#concept_pby_dh2_lpb">deprecated</a> and may be removed in a future release.</span>
                </div></div>
        <p class="- topic/p p">The Spark Evaluator processor is especially useful when you need to perform heavy custom
            processing within a pipeline, such as image classification.</p>
        <div class="- topic/p p">Complete the following general tasks to use the Spark Evaluator:<ol class="- topic/ol ol" id="concept_cpx_1lm_zx__ol_pmh_33p_zx" data-ofbid="concept_cpx_1lm_zx__ol_pmh_33p_zx">
                <li class="- topic/li li">Write the Spark application using Java or Scala. Then, package a JAR file
                    containing the application.</li>
                <li class="- topic/li li">Install the application and its dependencies.</li>
                <li class="- topic/li li">Configure the Spark Evaluator processor to submit the Spark application.<p class="- topic/p p">When
                        you configure the processor, you define the name of the custom Spark class
                        that you developed and define the arguments to pass to the
                            <code class="+ topic/ph pr-d/codeph ph codeph">init</code> method in the custom Spark class.</p></li>
            </ol><div class="- topic/note note note note_note"><span class="note__title">Note:</span> The stage libraries that include the Spark Evaluator include all dependencies
                required for the processor. The Spark Evaluator does not require any installation
                prerequisites.</div></div>
    </div>
<article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title2" id="concept_nbj_1jb_c1b">
 <h2 class="- topic/title title topictitle2" id="ariaid-title2">Spark Versions and Stage Libraries</h2>
 <div class="- topic/body concept/conbody body conbody">
  <p class="- topic/p p">The Spark Evaluator processor supports only <span class="- topic/ph ph">Spark version 2.1 or later</span>. </p>
        <div class="- topic/p p">When you use the Spark Evaluator processor, make sure the Spark version is the same
            across all related components, as follows:<ul class="- topic/ul ul" id="concept_nbj_1jb_c1b__ul_yct_nqb_c1b" data-ofbid="concept_nbj_1jb_c1b__ul_yct_nqb_c1b">
                <li class="- topic/li li">
                    <p class="- topic/p p">Make sure <span class="- topic/ph ph">the Spark version used in the selected stage
                        library matches the Spark version used to build the application.</span></p>
                    <p class="- topic/p p">For example, if you use Spark 2.1 to build the application, use a Spark
                        Evaluator provided in one of the Spark 2.1 stage libraries. </p>
                </li>
                <li class="- topic/li li"><p class="- topic/p p">When using the processor <span class="- topic/ph ph">in a cluster streaming pipeline, the Spark version in
                        the selected stage library must also match the Spark version used by the
                        cluster.</span></p><span class="- topic/ph ph">For example, if your cluster uses Spark 2.2, use a
                        stage library that includes Spark 2.2.</span></li>
            </ul></div>
        <p class="- topic/p p">The Spark Evaluator is <span class="- topic/ph ph">available in Cloudera and MapR stage libraries. To verify the
                        Spark version that a stage library includes, see the Cloudera or <span class="- topic/ph ph">HPE Ezmeral Data Fabric</span>
                        documentation. For more information about the stage libraries that include
                        the stage, see <a class="- topic/xref xref" href="../Installation/AddtionalStageLibs.html#concept_evs_xkm_s5">Available Stage Libraries</a>.</span></p>
 </div>
</article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title3" id="concept_hpm_tns_hz">
 <h2 class="- topic/title title topictitle2" id="ariaid-title3">Standalone Pipelines</h2>
 <div class="- topic/body concept/conbody body conbody">
        <p class="- topic/p p">When used in a standalone pipeline, the Spark
            Evaluator processor starts a local Spark application. It passes a batch of data to the
            Spark application as a Resilient Distributed Dataset (RDD). The Spark application - or
                <code class="+ topic/ph pr-d/codeph ph codeph">SparkContext</code> - runs as long as the pipeline runs. The Spark
            application submits jobs to the StreamSets Spark Transformer API, processing the data
            and then returning the results and errors back to the pipeline for further processing. </p>
        <p class="- topic/p p">When you use the Spark Evaluator in a standalone pipeline, define a parallelism value for
            the Spark Evaluator. The Spark application creates this number of partitions for each
            batch of records. The Spark Transformer processes each partition in parallel, and then
            returns the results and errors back to the pipeline.</p>
        <div class="- topic/note note note note_note"><span class="note__title">Note:</span> When you write your custom Spark application for use in a standalone pipeline, do not
            use the <code class="+ topic/ph pr-d/codeph ph codeph">RDD.checkpoint()</code> method. Checkpointing RDDs when the Spark
            Evaluator is in a standalone pipeline can cause the pipeline to fail.</div>
 </div>
</article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title4" id="concept_b1l_5lv_jz">
    <h2 class="- topic/title title topictitle2" id="ariaid-title4">Cluster Pipelines</h2>
    
    <div class="- topic/body concept/conbody body conbody"><p class="- topic/shortdesc shortdesc"></p>
        <p class="- topic/p p">You can use the Spark Evaluator processor in cluster
            pipelines that process data from a Kafka or MapR cluster in cluster streaming mode. You
            cannot use the processor in cluster pipelines that process data from a MapR cluster in
            cluster batch mode or from HDFS.</p>
        <p class="- topic/p p"> In a <a class="- topic/xref xref" href="../Cluster_Mode/ClusterPipelines.html#concept_hmh_kfn_1s">cluster pipeline</a>, the cluster manager spawns a <span class="- topic/ph ph">Data Collector</span>
            worker for each topic partition in the Kafka or MapR cluster. Spark Streaming generates
            a batch for the pipeline every few seconds. Then each <span class="- topic/ph ph">Data Collector</span>
            worker processes the batches from a single partition.</p>
        <p class="- topic/p p">Resilient Distributed Datasets (RDDs) are generated from data received across all of the
                <span class="- topic/ph ph">Data Collector</span>
            workers. The RDDs are then passed to the Spark Transformer running on the driver.</p>
        <p class="- topic/p p">The RDD passed to the Spark Transformer points to data across the cluster. The Spark
            Transformer processes all of the data received across all <span class="- topic/ph ph">Data Collector</span>
            workers in the cluster, processing multiple batches at the same time. Data moves from
            one node to another only if your custom Spark code shuffles the data. The Spark
            Transformer returns the results and errors back to each Spark Evaluator processor
            running on the <span class="- topic/ph ph">Data Collector</span>
            workers. The <span class="- topic/ph ph">Data Collector</span>
            workers process the remaining pipeline stages in parallel.</p>
        <p class="- topic/p p">The following image shows how the Spark Transformer processes data received across all
            workers in the cluster:</p>
        <p class="- topic/p p"><img class="- topic/image image" id="concept_b1l_5lv_jz__image_swx_xwv_jz" src="../Graphics/SparkEvaluatorCluster.png" height="204" width="826"/></p>
        <div class="- topic/p p">Use the following rules and guidelines when you write the Spark application and configure
            the processor for a cluster pipeline:<ul class="- topic/ul ul" id="concept_b1l_5lv_jz__ul_dg5_nmv_jz" data-ofbid="concept_b1l_5lv_jz__ul_dg5_nmv_jz">
                <li class="- topic/li li">Ensure that the Spark version you use to build the application matches the Spark
                    version used by the cluster and the Spark Evaluator stage library.</li>
                <li class="- topic/li li">To maintain the states of the RDDs that are used more than once in the
                    application, use the <code class="+ topic/ph pr-d/codeph ph codeph">RDD.checkpoint()</code> or
                        <code class="+ topic/ph pr-d/codeph ph codeph">RDD.cache()</code> method in your custom Spark application. When
                    you checkpoint or cache the RDDs, the Spark Transformer can access the RDDs from
                    previous batches.<p class="- topic/p p">The Spark Evaluator processor cleans up checkpoints when the
                        pipeline restarts or when the Spark application restarts. To maintain the
                        state of each RDD between pipeline runs or between restarts of the Spark
                        application, you must maintain the RDDs in an external store such as HDFS or
                        HBase.</p></li>
                <li class="- topic/li li">When you configure the processor, do not define a parallelism value for the
                    processor. The number of partitions defined in the Kafka Consumer origin
                    determines the number of partitions used throughout the cluster pipeline.</li>
                <li class="- topic/li li">When you configure the processor, use constant values for <code class="+ topic/ph pr-d/codeph ph codeph">init</code>
                    method arguments. In a cluster pipeline, the Spark Evaluator processor cannot
                    evaluate <span class="- topic/ph ph">Data Collector</span> expressions in <code class="+ topic/ph pr-d/codeph ph codeph">init</code> method arguments.</li>
                <li class="- topic/li li">When you monitor the running pipeline, the record throughput for
                    intervals less than 15 minutes might show an inaccurate count. To accurately
                    monitor the throughput rate, evaluate the throughput for 15 minutes or
                    more.</li>
            </ul></div>
    </div>
</article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title5" id="concept_lfl_dvd_1y">
    <h2 class="- topic/title title topictitle2" id="ariaid-title5">Developing the Spark Application</h2>
    <div class="- topic/body concept/conbody body conbody">
        <p class="- topic/p p">To develop a custom Spark
            application, you write the Spark application and then package a JAR file containing the
            application.</p>
        <p class="- topic/p p">Use Java or Scala to write a custom Spark class that implements the StreamSets Spark
            Transformer API: <a class="- topic/xref xref" href="https://github.com/streamsets/datacollector-plugin-api-oss/tree/master/streamsets-datacollector-spark-api" target="_blank" rel="external noopener">https://github.com/streamsets/datacollector-plugin-api-oss/tree/master/streamsets-datacollector-spark-api</a>.</p>
        <p class="- topic/p p">Include the following methods in the custom class:</p>
        <dl class="- topic/dl dl">
            
                <dt class="- topic/dt dt dlterm">init</dt>
                <dd class="- topic/dd dd">Optional. The <code class="+ topic/ph pr-d/codeph ph codeph">init</code> method is called once when the pipeline
                    starts to read arguments that you configure in the Spark Evaluator processor.
                    Use the <code class="+ topic/ph pr-d/codeph ph codeph">init</code> method to make connections to external systems or
                    to read configuration details or existing data from external systems.</dd>
            
            
                <dt class="- topic/dt dt dlterm">transform</dt>
                <dd class="- topic/dd dd">Required. The <code class="+ topic/ph pr-d/codeph ph codeph">transform</code> method is called for each batch of
                    records that the pipeline processes. The Spark Evaluator processor passes a
                    batch of data to the <code class="+ topic/ph pr-d/codeph ph codeph">transform</code> method as a Resilient
                    Distributed Dataset (RDD). The method processes the data according to the custom
                    code.</dd>
            
            
                <dt class="- topic/dt dt dlterm">destroy</dt>
                <dd class="- topic/dd dd">Optional. If you include an <code class="+ topic/ph pr-d/codeph ph codeph">init</code> method that makes connections
                    to external systems, you should also include the <code class="+ topic/ph pr-d/codeph ph codeph">destroy</code> method
                    to close the connections. The <code class="+ topic/ph pr-d/codeph ph codeph">destroy</code> method is called when the
                    pipeline stops.</dd>
            
        </dl>
        <p class="- topic/p p">When you finish writing the custom Spark class, package a JAR file containing the Spark
            application. Compile against the same stage library version that you use for the Spark
            Evaluator processor. For example, if you are using the Spark Evaluator processor
            included in the stage library for the Cloudera CDH version 5.9 distribution of Hadoop,
            build the application against Spark integrated into Cloudera CDH version 5.9.</p>
        <p class="- topic/p p">If you used Scala to write the custom Spark class, compile the application with Scala
            2.11.</p>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title6" id="task_dr2_gvd_1y">
    <h2 class="- topic/title title topictitle2" id="ariaid-title6">Installing the Application</h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc">Install the Spark application JAR file as an external library for <span class="- topic/ph ph">Data Collector</span>. If
        your custom Spark application depends on external libraries other than the <span class="- topic/ph ph">streamsets-datacollector-api,
                        streamsets-datacollector-spark-api, and spark-core libraries</span>, install those libraries in the same location as well. </p>
        <section class="- topic/section task/context section context"><div class="- topic/note note tip note_tip"><span class="note__title">Tip:</span> To include all dependent libraries used in your custom Spark
                application, you can use the Apache Maven Dependency Plugin. For more information
                about the Dependency Plugin, see <a class="- topic/xref xref" href="http://maven.apache.org/plugins/maven-dependency-plugin/" target="_blank" rel="external noopener">http://maven.apache.org/plugins/maven-dependency-plugin/</a>.</div><span class="- topic/ph ph">For information about installing additional drivers, <span class="- topic/ph ph" id="task_dr2_gvd_1y__SDCDPM-ExtLibs-linksOnly">see <a class="- topic/xref xref" href="../Configuration/ExternalLibs.html#concept_pdv_qlw_ft">Install External Libraries</a>.</span></span><div class="- topic/p p">
                <div class="- topic/note note note note_note"><span class="note__title">Note:</span> The Spark Evaluator processor is included in several CDH and MapR stage
                    libraries. When you install external libraries for the processor, you must make
                    them accessible to the stage library that you are using.</div>
            </div></section>
    </div>
    
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title7" id="task_g1p_gqn_zx">
    <h2 class="- topic/title title topictitle2" id="ariaid-title7">Configuring a Spark Evaluator Processor</h2>
    <div class="- topic/body task/taskbody body taskbody">
        <section class="- topic/section task/context section context">
            <p class="- topic/p p">Configure a Spark Evaluator
                to process data based on a custom Spark application.</p>
        </section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the Properties panel, on the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">General</span> tab, configure the
                    following properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_g1p_gqn_zx__table_blh_n5h_ur" data-ofbid="task_g1p_gqn_zx__table_blh_n5h_ur" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_g1p_gqn_zx__table_blh_n5h_ur__entry__1">General Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_g1p_gqn_zx__table_blh_n5h_ur__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__1">Name</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__2">Stage name.</td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__1">Description</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__2">Optional description.</td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__1">Stage Library</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__2">Library version that you want to use. </td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__1"><a class="- topic/xref xref" href="../Pipeline_Design/DroppingUnwantedRecords.html#concept_dnj_bkm_vq">Required Fields</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__2">Fields that must include data for the record to be passed
                                        into the stage. <div class="- topic/note note tip note_tip"><span class="note__title">Tip:</span> You might
                                            include fields that the stage uses.</div><p class="- topic/p p">Records
                                            that do not include all required fields are processed
                                            based on the error handling configured for the
                                            pipeline.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__1"><a class="- topic/xref xref" href="../Pipeline_Design/DroppingUnwantedRecords.html#concept_msl_yd4_fs">Preconditions</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__2">Conditions that must evaluate to TRUE to allow a record
                                        to enter the stage for processing. Click
                                            <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Add</span> to create additional
                                        preconditions. <p class="- topic/p p">Records that do not meet all preconditions
                                            are processed based on the error handling configured for
                                            the stage.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__1"><a class="- topic/xref xref" href="../Pipeline_Design/ErrorHandling.html#concept_atr_j4y_5r">On Record Error</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_g1p_gqn_zx__table_blh_n5h_ur__entry__2">Error record handling for the stage: <ul class="- topic/ul ul" id="task_g1p_gqn_zx__ul_swr_ofh_hr" data-ofbid="task_g1p_gqn_zx__ul_swr_ofh_hr">
                                            <li class="- topic/li li">Discard - Discards the record.</li>
                                            <li class="- topic/li li">Send to Error - Sends the record to the pipeline for
                                                error handling.</li>
                                            <li class="- topic/li li">Stop Pipeline - Stops the pipeline. Not valid for
                                                cluster pipelines.</li>
                                        </ul></td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">On the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Spark</span> tab, configure the following
                    properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_g1p_gqn_zx__table_ufq_xf4_zx" data-ofbid="task_g1p_gqn_zx__table_ufq_xf4_zx" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_g1p_gqn_zx__table_ufq_xf4_zx__entry__1">Spark Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_g1p_gqn_zx__table_ufq_xf4_zx__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_g1p_gqn_zx__table_ufq_xf4_zx__entry__1"><a class="- topic/xref xref" href="Spark.html#concept_lfl_dvd_1y">Spark Transformer Class</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_g1p_gqn_zx__table_ufq_xf4_zx__entry__2">Fully qualified name of the custom Spark class that
                                        implements the StreamSets Spark Transformer API. Enter the
                                        class name using the following
                                            format:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>com.streamsets.spark.&lt;custom class&gt;</code></pre><div class="- topic/p p">For
                                            example, let's assume that you developed a
                                                <code class="+ topic/ph pr-d/codeph ph codeph">GetCreditCardType</code> class that
                                            implemented the Spark Transformer API as
                                            follows:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>public class GetCreditCardType extends SparkTransformer implements Serializable {
...
}</code></pre></div><div class="- topic/p p">Then
                                            you would enter the class name as
                                            follows:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>com.streamsets.spark.GetCreditCardType</code></pre></div></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_g1p_gqn_zx__table_ufq_xf4_zx__entry__1"><a class="- topic/xref xref" href="Spark.html#concept_lfl_dvd_1y">Init Method Arguments</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_g1p_gqn_zx__table_ufq_xf4_zx__entry__2">Arguments to pass to the <code class="+ topic/ph pr-d/codeph ph codeph">init</code> method in
                                        the custom Spark class. Enter arguments as required by your
                                        custom Spark class.<p class="- topic/p p">In standalone mode, enter a constant
                                            or an expression for the argument value. In cluster
                                            mode, enter constant values only. In a cluster pipeline,
                                            the Spark Evaluator processor cannot evaluate Data
                                            Collector expressions defined in <code class="+ topic/ph pr-d/codeph ph codeph">init</code>
                                            method arguments.</p><p class="- topic/p p">Using <a class="- topic/xref xref" href="../Pipeline_Configuration/SimpleBulkEdit.html#concept_alb_b3y_cbb">simple or bulk edit mode</a>, click the
                                                <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Add</span> icon to add additional
                                            arguments.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_g1p_gqn_zx__table_ufq_xf4_zx__entry__1">Parallelism</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_g1p_gqn_zx__table_ufq_xf4_zx__entry__2">For standalone mode, the number of partitions to create
                                        per batch of records. For example, if set to 4, then the
                                        Spark Transformer simultaneously runs 4 parallel jobs to
                                        process the batch. <p class="- topic/p p">Set the value based on the number of
                                            available processors on the <span class="- topic/ph ph">Data Collector</span> machine.</p><div class="- topic/note note note note_note"><span class="note__title">Note:</span> Not used when the processor is
                                            included in a cluster pipeline. The Spark Transformer
                                            uses the number of partitions defined in the Kafka
                                            Consumer or MapR Streams Consumer origin.</div></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_g1p_gqn_zx__table_ufq_xf4_zx__entry__1">Application Name</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_g1p_gqn_zx__table_ufq_xf4_zx__entry__2">For standalone mode, the name of the Spark application.
                                        Spark displays this application name in the log files. <p class="- topic/p p">If
                                            you run pipelines that include multiple Spark Evaluator
                                            processors, be sure to use a unique application name for
                                            each to make debugging simpler.</p><div class="- topic/p p">Default is "SDC
                                            Spark App". <div class="- topic/note note note note_note"><span class="note__title">Note:</span> Not used when the processor is
                                                included in a cluster pipeline.</div></div></td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li></ol></section>
    </div>
</article></article></article></main></div>

                        
                        
                        


                    </div>
                    
                </div>
            </div>


        </div> <nav class="navbar navbar-default wh_footer" data-whc_version="25.0">
  <div class=" footer-container  mx-auto">
    <!-- script for Data Collector, all flavors, but only used when accessed directly, not from portal --><script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-60917135-3', 'auto');
  ga('send', 'pageview');
</script>
  </div>
</nav>

        
        <div id="go2top">
            <span class="oxy-icon oxy-icon-up"></span>
        </div>
        
        <!-- The modal container for images -->
        <div id="modal_img_large" class="modal">
            <span class="close oxy-icon oxy-icon-remove"></span>
            <!-- Modal Content (The Image) -->
            <div id="modal_img_container"></div>
            <!-- Modal Caption (Image Text) -->
            <div id="caption"></div>
        </div>
        
        
        Â© 2023 StreamSets, Inc.

    </body>
</html>